import os, sys

now_dir = os.getcwd()
sys.path.append(now_dir)
sys.path.append(os.path.join(now_dir, "GPT_SoVITS"))
os.environ["OPENAI_API_KEY"] = "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"
os.environ["SERPAPI_API_KEY"] = "9200796cba4b2569d70549a440e2ee16c690401256613ec3cdeffc46edb47652"

from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_openai import OpenAIEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma

splits = []

import os
import time


def get_all_files():
    file_paths = []
    for root, dirs, files in os.walk("D:\\\\workspace\\git-clone\\interview-for-vectorstore"):
        for file in files:
            file_extension = file.split(".")[-1]
            if file_extension == "md":
                file_paths.append(os.path.join(root, file))
    return file_paths


files = get_all_files()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
for file in files:
    loader = UnstructuredMarkdownLoader(file)
    docs = loader.load()
    splits += text_splitter.split_documents(docs)
print(len(splits))

i = 0
j = 0
while i + 100 < 3744:
    print(j)
    vectorstore = Chroma.from_documents(documents=splits[i:i + 100], embedding=OpenAIEmbeddings(),
                                        persist_directory=f"./chroma_db{j}")
    time.sleep(60)
    i += 100
    j += 1
